{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ae460052",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "501b91a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, TFBertModel, TFBertForMaskedLM, BertForMaskedLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "55d13fa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint-136500  checkpoint-137500  checkpoint-138500\r\n",
      "checkpoint-137000  checkpoint-138000  runs\r\n"
     ]
    }
   ],
   "source": [
    "!ls bert-multilanguage-8layers-mlm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "11cc49e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('./bert-multilanguage-8layers-mlm/checkpoint-130000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6dac7a1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BertForMaskedLM.from_pretrained('./bert-multilanguage-8layers-mlm/checkpoint-130000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7b8a3fce",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/utils/_deprecation.py:39: FutureWarning: Pass token='finetuned-bert-base-multilingual-cased-noisy-en-ms' as keyword args. From version 0.7 passing these as positional arguments will result in an error\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:79: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.7. Pass `repo_id` instead.\n",
      "  warnings.warn(\n",
      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:596: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
      "  warnings.warn(\n",
      "Cloning https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms into local empty directory.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a16742d73f8b4a918cc20b7c4dbacc93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 4.00k/571M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
      "   20ab8ea..db66733  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms/commit/db66733a8cb6078dc20b7ffa5121c251117bbbbe'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('finetuned-bert-base-multilingual-cased-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a6ccfca1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
      "   db66733..5962025  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms/commit/5962025946a68ab1fc30ed2d55de5b6fea91f531'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('finetuned-bert-base-multilingual-cased-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3e343c6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-30 14:39:46.112266: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
      "2022-06-30 14:39:46.112284: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: huseincomel-desktop\n",
      "2022-06-30 14:39:46.112288: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: huseincomel-desktop\n",
      "2022-06-30 14:39:46.112326: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: Not found: was unable to find libcuda.so DSO loaded into this program\n",
      "2022-06-30 14:39:46.112346: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 470.129.6\n",
      "2022-06-30 14:39:46.112596: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-06-30 14:39:46.369163: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 367248384 exceeds 10% of free system memory.\n",
      "2022-06-30 14:39:46.516846: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 367248384 exceeds 10% of free system memory.\n",
      "2022-06-30 14:39:46.558557: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 367248384 exceeds 10% of free system memory.\n",
      "2022-06-30 14:39:46.941689: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 367248384 exceeds 10% of free system memory.\n",
      "2022-06-30 14:39:47.061699: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 367248384 exceeds 10% of free system memory.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForMaskedLM: ['cls.predictions.decoder.bias', 'bert.embeddings.position_ids']\n",
      "- This IS expected if you are initializing TFBertForMaskedLM from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertForMaskedLM from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertForMaskedLM were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForMaskedLM for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "tf_model = TFBertForMaskedLM.from_pretrained('./bert-multilanguage-8layers-mlm/checkpoint-130000',\n",
    "                                          from_pt = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8cceb08f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed82b5f5017f442f9557e9fccec6ca79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file tf_model.h5:   0%|          | 4.00k/923M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
      "   5962025..c588914  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms/commit/c5889145ccf27f15b5c87c66cd5f95a26d9b3839'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_model.push_to_hub('finetuned-bert-base-multilingual-cased-noisy-en-ms', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "956cb1ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'finetuned-bert-base-multilingual-cased-noisy-en-ms' already exists and is not an empty directory.\n",
      "Already up to date.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
    "!cd finetuned-bert-base-multilingual-cased-noisy-en-ms && git pull"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bed0dcf1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[main 84da4e7] add tensorboard\n",
      " 12 files changed, 36 insertions(+)\n",
      " create mode 100644 runs/Jun30_00-38-34_huseincomel-desktop/1656520857.8115973/events.out.tfevents.1656520857.huseincomel-desktop.680534.1\n",
      " create mode 100644 runs/Jun30_00-38-34_huseincomel-desktop/events.out.tfevents.1656520857.huseincomel-desktop.680534.0\n",
      " create mode 100644 runs/Jun30_00-49-58_huseincomel-desktop/1656521409.1117074/events.out.tfevents.1656521409.huseincomel-desktop.1097430.1\n",
      " create mode 100644 runs/Jun30_00-49-58_huseincomel-desktop/events.out.tfevents.1656521409.huseincomel-desktop.1097430.0\n",
      " create mode 100644 runs/Jun30_00-50-23_huseincomel-desktop/1656521431.54345/events.out.tfevents.1656521431.huseincomel-desktop.1112907.1\n",
      " create mode 100644 runs/Jun30_00-50-23_huseincomel-desktop/events.out.tfevents.1656521431.huseincomel-desktop.1112907.0\n",
      " create mode 100644 runs/Jun30_01-19-47_huseincomel-desktop/1656523880.0961938/events.out.tfevents.1656523880.huseincomel-desktop.2193252.1\n",
      " create mode 100644 runs/Jun30_01-19-47_huseincomel-desktop/events.out.tfevents.1656523880.huseincomel-desktop.2193252.0\n",
      " create mode 100644 runs/Jun30_09-09-40_huseincomel-desktop/1656551774.1034188/events.out.tfevents.1656551774.huseincomel-desktop.6848.1\n",
      " create mode 100644 runs/Jun30_09-09-40_huseincomel-desktop/events.out.tfevents.1656551774.huseincomel-desktop.6848.0\n",
      " create mode 100644 runs/Jun30_09-17-10_huseincomel-desktop/1656551839.6757455/events.out.tfevents.1656551839.huseincomel-desktop.7932.1\n",
      " create mode 100644 runs/Jun30_09-17-10_huseincomel-desktop/events.out.tfevents.1656551839.huseincomel-desktop.7932.0\n",
      "Uploading LFS objects: 100% (12/12), 92 KB | 27 KB/s, done.                     \n",
      "Enumerating objects: 28, done.\n",
      "Counting objects: 100% (28/28), done.\n",
      "Delta compression using up to 16 threads\n",
      "Compressing objects: 100% (27/27), done.\n",
      "Writing objects: 100% (27/27), 3.16 KiB | 3.16 MiB/s, done.\n",
      "Total 27 (delta 1), reused 0 (delta 0)\n",
      "To https://huggingface.co/mesolitica/finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
      "   c588914..84da4e7  main -> main\n"
     ]
    }
   ],
   "source": [
    "!cp -r bert-multilanguage-8layers-mlm/runs finetuned-bert-base-multilingual-cased-noisy-en-ms\n",
    "!cd finetuned-bert-base-multilingual-cased-noisy-en-ms && git add . && git commit -m 'add tensorboard' && git push"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
